{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import requests\n",
    "import json\n",
    "import hashlib\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "password_map = {'正确': 0, '错误': 1, 'A': 0, 'B': 1, 'C': 2, 'D': 3}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_page = 9999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "session = requests.session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    'xuehao': '',  # TODO your student number\n",
    "    'password': 'ca963147',  # # TODO the website password\n",
    "    'role': '0',\n",
    "    'postflag': '1',\n",
    "    'catalog_id': '13404',\n",
    "    'cmd': 'login'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subject_format(subject: str) -> str:\n",
    "    \"\"\"\n",
    "    '279、气体钢瓶使用后，可以不关闭阀门。' -> '气体钢瓶使用后，可以不关闭阀门。'\n",
    "    \"\"\"\n",
    "    return re.match('\\d+、(.*)', subject).group(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "def password_format(password: str):\n",
    "    \"\"\"\n",
    "    '（标准答案：正确）' -> 0\n",
    "    '（标准答案：错误）' -> 1\n",
    "    '（标准答案：A）' -> 0\n",
    "    '（标准答案：B）' -> 1\n",
    "    '（标准答案：C）' -> 2\n",
    "    '（标准答案：D）' -> 3\n",
    "    '（标准答案：ABC）' -> [0, 1, 2]\n",
    "    \"\"\"\n",
    "    password = password.strip().replace('\\t', '').replace('\\n', '')\n",
    "    value = re.match('（标准答案：([\\u2E80-\\u9FFF]+|[A-D]+)）', password).group(1)\n",
    "    if value in password_map.keys():\n",
    "        # 单选\n",
    "        return password_map[value]\n",
    "    else:\n",
    "        # 多选\n",
    "        result = []\n",
    "        for v in value:\n",
    "            result.append(password_map[v])\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session.post('http://10.10.10.46/exam_login.php', data=data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "def section_crawler(fn: str, tikubh: str):\n",
    "    result_list = []\n",
    "    for i in range(1, max_page):\n",
    "        resp = session.get('http://10.10.10.46/redir.php?catalog_id=6&cmd=learning&tikubh={0}&page={1}'\n",
    "                           .format(tikubh, i))\n",
    "        resp.encoding = 'gbk'\n",
    "        sp = BeautifulSoup(resp.text, 'lxml')\n",
    "\n",
    "        subject_list = sp.select('div.shiti-content div.shiti')\n",
    "        password_list = sp.select('div.shiti-content span')\n",
    "        if len(subject_list) == 0:\n",
    "            break\n",
    "\n",
    "        print('Page %d OK' % i)\n",
    "\n",
    "        for i in range(len(subject_list)):\n",
    "            title = subject_format(subject_list[i].select_one('h3').get_text(strip=True))\n",
    "            # title_md5 = hashlib.sha256(title.encode('utf-8')).hexdigest()\n",
    "            password = password_format(password_list[i].get_text(strip=True))\n",
    "            result_list.append({'title': title, 'password': password})\n",
    "\n",
    "    with open('./question_bank_{0}.json'.format(fn), 'w', encoding='utf-8') as fp:\n",
    "        json.dump(result_list, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = [\n",
    "    {\n",
    "        'type': '化学类安全题',\n",
    "        'tikubh': '1436'\n",
    "    }, {\n",
    "        'type': '医学生物类安全题',\n",
    "        'tikubh': '1467'\n",
    "    }, {\n",
    "        'type': '通识类安全题',\n",
    "        'tikubh': '1471'\n",
    "    }, {\n",
    "        'type': '机械建筑类安全题',\n",
    "        'tikubh': '1484'\n",
    "    }, {\n",
    "        'type': '电气类安全题',\n",
    "        'tikubh': '1485'\n",
    "    }, {\n",
    "        'type': '辐射类安全题',\n",
    "        'tikubh': '1486'\n",
    "    }, {\n",
    "        'type': '特种设备安全题',\n",
    "        'tikubh': '4199'\n",
    "    }, {\n",
    "        'type': '消防安全题',\n",
    "        'tikubh': '4200'\n",
    "    }, {\n",
    "        'type': 'test',\n",
    "        'tikubh': '12263'\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "化学类安全题\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "Page 11 OK\n",
      "Page 12 OK\n",
      "Page 13 OK\n",
      "Page 14 OK\n",
      "Page 15 OK\n",
      "Page 16 OK\n",
      "Page 17 OK\n",
      "Page 18 OK\n",
      "Page 19 OK\n",
      "Page 20 OK\n",
      "Page 21 OK\n",
      "Page 22 OK\n",
      "Page 23 OK\n",
      "Page 24 OK\n",
      "Page 25 OK\n",
      "Page 26 OK\n",
      "Page 27 OK\n",
      "Page 28 OK\n",
      "Page 29 OK\n",
      "Page 30 OK\n",
      "Page 31 OK\n",
      "Page 32 OK\n",
      "Page 33 OK\n",
      "Page 34 OK\n",
      "Page 35 OK\n",
      "Page 36 OK\n",
      "Page 37 OK\n",
      "Page 38 OK\n",
      "Page 39 OK\n",
      "Page 40 OK\n",
      "Page 41 OK\n",
      "Page 42 OK\n",
      "Page 43 OK\n",
      "Page 44 OK\n",
      "Page 45 OK\n",
      "Page 46 OK\n",
      "Page 47 OK\n",
      "Page 48 OK\n",
      "Page 49 OK\n",
      "Page 50 OK\n",
      "Page 51 OK\n",
      "Page 52 OK\n",
      "Page 53 OK\n",
      "Page 54 OK\n",
      "Page 55 OK\n",
      "Page 56 OK\n",
      "Page 57 OK\n",
      "Page 58 OK\n",
      "Page 59 OK\n",
      "Page 60 OK\n",
      "Page 61 OK\n",
      "Page 62 OK\n",
      "Page 63 OK\n",
      "Page 64 OK\n",
      "Page 65 OK\n",
      "Page 66 OK\n",
      "Page 67 OK\n",
      "Page 68 OK\n",
      "Page 69 OK\n",
      "Page 70 OK\n",
      "Page 71 OK\n",
      "Page 72 OK\n",
      "Page 73 OK\n",
      "Page 74 OK\n",
      "Page 75 OK\n",
      "Page 76 OK\n",
      "Page 77 OK\n",
      "Page 78 OK\n",
      "Page 79 OK\n",
      "Page 80 OK\n",
      "Page 81 OK\n",
      "Page 82 OK\n",
      "Page 83 OK\n",
      "Page 84 OK\n",
      "医学生物类安全题\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "Page 11 OK\n",
      "Page 12 OK\n",
      "Page 13 OK\n",
      "Page 14 OK\n",
      "Page 15 OK\n",
      "Page 16 OK\n",
      "Page 17 OK\n",
      "Page 18 OK\n",
      "Page 19 OK\n",
      "Page 20 OK\n",
      "Page 21 OK\n",
      "Page 22 OK\n",
      "Page 23 OK\n",
      "Page 24 OK\n",
      "Page 25 OK\n",
      "Page 26 OK\n",
      "Page 27 OK\n",
      "Page 28 OK\n",
      "Page 29 OK\n",
      "Page 30 OK\n",
      "Page 31 OK\n",
      "Page 32 OK\n",
      "Page 33 OK\n",
      "Page 34 OK\n",
      "Page 35 OK\n",
      "Page 36 OK\n",
      "Page 37 OK\n",
      "通识类安全题\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "Page 11 OK\n",
      "Page 12 OK\n",
      "Page 13 OK\n",
      "Page 14 OK\n",
      "Page 15 OK\n",
      "Page 16 OK\n",
      "Page 17 OK\n",
      "Page 18 OK\n",
      "Page 19 OK\n",
      "Page 20 OK\n",
      "Page 21 OK\n",
      "Page 22 OK\n",
      "Page 23 OK\n",
      "Page 24 OK\n",
      "Page 25 OK\n",
      "Page 26 OK\n",
      "Page 27 OK\n",
      "Page 28 OK\n",
      "Page 29 OK\n",
      "Page 30 OK\n",
      "Page 31 OK\n",
      "Page 32 OK\n",
      "Page 33 OK\n",
      "Page 34 OK\n",
      "Page 35 OK\n",
      "Page 36 OK\n",
      "Page 37 OK\n",
      "Page 38 OK\n",
      "Page 39 OK\n",
      "Page 40 OK\n",
      "Page 41 OK\n",
      "Page 42 OK\n",
      "Page 43 OK\n",
      "Page 44 OK\n",
      "Page 45 OK\n",
      "Page 46 OK\n",
      "Page 47 OK\n",
      "Page 48 OK\n",
      "Page 49 OK\n",
      "Page 50 OK\n",
      "Page 51 OK\n",
      "Page 52 OK\n",
      "Page 53 OK\n",
      "Page 54 OK\n",
      "Page 55 OK\n",
      "Page 56 OK\n",
      "Page 57 OK\n",
      "Page 58 OK\n",
      "Page 59 OK\n",
      "Page 60 OK\n",
      "Page 61 OK\n",
      "Page 62 OK\n",
      "Page 63 OK\n",
      "Page 64 OK\n",
      "Page 65 OK\n",
      "Page 66 OK\n",
      "Page 67 OK\n",
      "Page 68 OK\n",
      "Page 69 OK\n",
      "Page 70 OK\n",
      "Page 71 OK\n",
      "Page 72 OK\n",
      "Page 73 OK\n",
      "Page 74 OK\n",
      "Page 75 OK\n",
      "Page 76 OK\n",
      "Page 77 OK\n",
      "Page 78 OK\n",
      "Page 79 OK\n",
      "Page 80 OK\n",
      "机械建筑类安全题\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "Page 11 OK\n",
      "Page 12 OK\n",
      "Page 13 OK\n",
      "Page 14 OK\n",
      "Page 15 OK\n",
      "Page 16 OK\n",
      "Page 17 OK\n",
      "Page 18 OK\n",
      "Page 19 OK\n",
      "Page 20 OK\n",
      "Page 21 OK\n",
      "Page 22 OK\n",
      "Page 23 OK\n",
      "Page 24 OK\n",
      "Page 25 OK\n",
      "Page 26 OK\n",
      "Page 27 OK\n",
      "电气类安全题\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "Page 11 OK\n",
      "Page 12 OK\n",
      "Page 13 OK\n",
      "Page 14 OK\n",
      "Page 15 OK\n",
      "Page 16 OK\n",
      "Page 17 OK\n",
      "Page 18 OK\n",
      "Page 19 OK\n",
      "辐射类安全题\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "Page 11 OK\n",
      "Page 12 OK\n",
      "特种设备安全题\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "消防安全题\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "Page 11 OK\n",
      "Page 12 OK\n",
      "Page 13 OK\n",
      "Page 14 OK\n",
      "Page 15 OK\n",
      "Page 16 OK\n",
      "test\n",
      "Page 1 OK\n",
      "Page 2 OK\n",
      "Page 3 OK\n",
      "Page 4 OK\n",
      "Page 5 OK\n",
      "Page 6 OK\n",
      "Page 7 OK\n",
      "Page 8 OK\n",
      "Page 9 OK\n",
      "Page 10 OK\n",
      "Page 11 OK\n",
      "Page 12 OK\n",
      "Page 13 OK\n",
      "Page 14 OK\n",
      "Page 15 OK\n",
      "Page 16 OK\n",
      "Page 17 OK\n",
      "Page 18 OK\n",
      "Page 19 OK\n",
      "Page 20 OK\n",
      "Page 21 OK\n",
      "Page 22 OK\n",
      "Page 23 OK\n",
      "Page 24 OK\n",
      "Page 25 OK\n",
      "Page 26 OK\n",
      "Page 27 OK\n",
      "Page 28 OK\n",
      "Page 29 OK\n",
      "Page 30 OK\n",
      "Page 31 OK\n",
      "Page 32 OK\n",
      "Page 33 OK\n",
      "Page 34 OK\n",
      "Page 35 OK\n",
      "Page 36 OK\n",
      "Page 37 OK\n",
      "Page 38 OK\n",
      "Page 39 OK\n",
      "Page 40 OK\n"
     ]
    }
   ],
   "source": [
    "for p in params:\n",
    "    print(p['type'])\n",
    "    section_crawler(p['type'], p['tikubh'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
